/**
 * Project  : Looker
 * Package  : net.freestu.looker.monitor
 * File     : HtmlFetcher.java
 * Author   : solosky < solosky772@qq.com >
 * Date     : Aug 23, 2009
 * Modified : Aug 23, 2009
 * License  : Apache License 2.0 
 */
package net.freestu.looker.monitor;

import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;

import org.apache.log4j.Logger;
import org.mozilla.intl.chardet.nsDetector;
import org.mozilla.intl.chardet.nsICharsetDetectionObserver;

/**
 *  网页下载器
 *  下载html文档
 */
public class HtmlFetcher extends AbstractFetcher
{

	/**
	 * 字符编码
	 */
	private String charset;
	
	/**
	 * 下载的HTML源代码
	 */
	 private String html;
	 
	 /**
	  * 日志记录
	  */
	
	private static Logger logger = Logger.getLogger(FileFetcher.class);
	
	/**
	 * 只有一个地址参数的构造函数
     * @param surl
     * @throws FetcherException
     */
    public HtmlFetcher(String surl) throws FetcherException
    {
	    super(surl);
    }
    
    /**
     * 有地址和闭锁对象的构造函数
     * @param surl
     * @param latch
     * @throws FetcherException
     */
    public HtmlFetcher(String surl, CountDownLatch latch) throws FetcherException
    {
	    super(surl,latch);
    }

	/* (non-Javadoc)
     * @see net.freestu.looker.monitor.AbstractFetcher#fetch()
     */
    @Override
    public void fetch() throws FetcherException
    {
    	logger.debug("开始下载网页:" +url);
	   openConnection();
	   fetchHtml();
	    
    }

	/* (non-Javadoc)
     * @see net.freestu.looker.monitor.AbstractFetcher#getFeched()
     */
    @Override
    public Object getFeched()
    {
	   return this.html;
    }
    
    /**
	 * 返回此URL的网页内容
	 * 去除了空白字符，便于后面的正则表达式提取信息
	 * @return
     * @throws FetcherException 
	 * @throws IOException
	 */
	public void fetchHtml() throws FetcherException
	{
		try {
    		BufferedReader r = new BufferedReader(
    							new InputStreamReader(
    									connection.getInputStream(),
    									guessCharset(url.openStream())));
    		StringBuffer s = new StringBuffer();
    		String line = null;
    		while ( (line = r.readLine())!=null )
    		{
    			//去除空白字符 TODO 可以用正则表达式去除 。。。
    			line = line.replaceAll("\t", "");
    			line = line.replaceAll("\r", "");
    			line = line.replaceAll("\n", "");
    			s.append(line);
    		}
    		html = s.toString();
		}catch (Exception e) {
			throw new FetcherException(e.getMessage());
		}
	}
	
    
    /**
     * 猜测网页编码
     * 使用jchardet来猜测，准确率很高
     * @throws IOException 
     */
    private String guessCharset(InputStream in) throws IOException
    {
    	BufferedInputStream imp = new BufferedInputStream(in);	    
		 nsDetector det = new nsDetector(nsDetector.ALL) ;
		    // 设置一个Oberver
		    // 实现nsICharsetDetectionObserver接口，这个接口只有一个Notify()方法.当jchardet引擎自己认为已经识别出字符串的字符集后(不论识别的对错)，都会调用这个Notify方法。
		    det.Init(new nsICharsetDetectionObserver(){
		        public void Notify(String c){
		             charset = c;
		        }
		      });
		    
	    byte[] buf = new byte[1024] ;
	    int len    = 1024;
	    boolean done = false ;//是否已经确定某种字符集
	    boolean isAscii = true ;//假定当前的串是ASCII编码
	       
	    while( (len=imp.read(buf,0,buf.length)) != -1) {
	    	 if (isAscii)
		            isAscii = det.isAscii(buf,len);

		        // 如果不是ascii字符，则调用DoIt方法.
		        if (!isAscii && !done)
		             done = det.DoIt(buf,len, false);//如果不是ASCII，又还没确定编码集，则继续检测。
	    }
	    det.DataEnd();
	    //如果没有成功猜测，默认为第一个最可能的字符集
	    if(charset==null) {
	    	String prob[] = det.getProbableCharsets();
	    	charset = prob.length>0 ? prob[0] : "UTF-8"; 
	    }
	    return charset;
    }
    
    public static void main(String[] args) throws FetcherException, InterruptedException, ExecutionException
    {
    	AbstractFetcher fc = new HtmlFetcher("http://www.baidu.com");
    	//fc.fetch();
    	//System.out.println((String) fc.getFeched());
    	
    	ExecutorService es = Executors.newCachedThreadPool();
    	Future<Object> f = es.submit(fc);
    	System.out.println((String)f.get());
    	es.shutdown();
    	
    }

}
